Skip to content

added sentry logs and better state management for ws #156

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
josancamon19 merged 1 commit into from
May 15, 2024
Merged

added sentry logs and better state management for ws #156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions apps/AppWithWearable/ios/Runner.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,7 @@
BUILD_LIBRARY_FOR_DISTRIBUTION = NO;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements;
CURRENT_PROJECT_VERSION = 4;
CURRENT_PROJECT_VERSION = 9;
DEVELOPMENT_TEAM = UA6Q3X7F6K;
ENABLE_BITCODE = NO;
FRAMEWORK_SEARCH_PATHS = (
Expand Down Expand Up @@ -510,7 +510,7 @@
BUILD_LIBRARY_FOR_DISTRIBUTION = NO;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements;
CURRENT_PROJECT_VERSION = 4;
CURRENT_PROJECT_VERSION = 9;
DEVELOPMENT_TEAM = UA6Q3X7F6K;
ENABLE_BITCODE = NO;
FRAMEWORK_SEARCH_PATHS = (
Expand Down Expand Up @@ -544,7 +544,7 @@
BUILD_LIBRARY_FOR_DISTRIBUTION = NO;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements;
CURRENT_PROJECT_VERSION = 4;
CURRENT_PROJECT_VERSION = 9;
DEVELOPMENT_TEAM = UA6Q3X7F6K;
ENABLE_BITCODE = NO;
FRAMEWORK_SEARCH_PATHS = (
Expand Down
1 change: 1 addition & 0 deletions apps/AppWithWearable/lib/pages/home/page.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ class _HomePageState extends State<HomePage> {
setState(() {
_device = null;
});
// Sentry.captureMessage('Friend Device Disconnected', level: SentryLevel.warning);
createNotification(title: 'Friend Device Disconnected', body: 'Please reconnect to continue using your Friend.');
scanAndConnectDevice().then((friendDevice) {
if (friendDevice != null) {
Expand Down
203 changes: 126 additions & 77 deletions apps/AppWithWearable/lib/pages/home/widgets/transcript.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ import 'dart:io';
import 'package:collection/collection.dart';
import 'package:flutter/material.dart';
import 'package:flutter/scheduler.dart';
import 'package:friend_private/backend/api_requests/api_calls.dart';
import 'package:friend_private/utils/memories.dart';
import 'package:friend_private/backend/api_requests/cloud_storage.dart';
import 'package:friend_private/utils/notifications.dart';
import 'package:friend_private/utils/sentry_log.dart';
import 'package:friend_private/utils/stt/deepgram.dart';
import 'package:friend_private/utils/stt/wav_bytes.dart';
import 'package:google_fonts/google_fonts.dart';
Expand All @@ -32,96 +33,49 @@ class TranscriptWidget extends StatefulWidget {
State<TranscriptWidget> createState() => TranscriptWidgetState();
}

class TranscriptWidgetState extends State<TranscriptWidget> with TickerProviderStateMixin {
List<Map<int, String>> whispersDiarized = [{}];
IOWebSocketChannel? channel;
class TranscriptWidgetState extends State<TranscriptWidget> with WidgetsBindingObserver {
WebsocketConnectionStatus wsConnectionState = WebsocketConnectionStatus.notConnected;
bool websocketReconnecting = false;
List<Map<int, String>> whispersDiarized = [{}];

IOWebSocketChannel? channel;
StreamSubscription? streamSubscription;
WavBytesUtil? audioStorage;

Timer? _timer;
Timer? _whisperTranscriptTimer;
Timer? _memoryCreationTimer;

String customWebsocketTranscript = '';
IOWebSocketChannel? channelCustomWebsocket;

String _buildDiarizedTranscriptMessage() {
int totalSpeakers = whispersDiarized
.map((e) => e.keys.isEmpty ? 0 : ((e.keys).max + 1))
.reduce((value, element) => value > element ? value : element);

debugPrint('Speakers count: $totalSpeakers');

String transcript = '';
for (int partIdx = 0; partIdx < whispersDiarized.length; partIdx++) {
var part = whispersDiarized[partIdx];
if (part.isEmpty) continue;
for (int speaker = 0; speaker < totalSpeakers; speaker++) {
if (part.containsKey(speaker)) {
// This part and previous have only 1 speaker, and is the same
if (partIdx > 0 &&
whispersDiarized[partIdx - 1].containsKey(speaker) &&
whispersDiarized[partIdx - 1].length == 1 &&
part.length == 1) {
transcript += '${part[speaker]!} ';
} else {
transcript += 'Speaker $speaker: ${part[speaker]!} ';
}
}
}
transcript += '\n';
}
return transcript;
}

_whisperTimer() {
_whisperTranscriptTimer?.cancel();
_whisperTranscriptTimer = Timer(const Duration(seconds: 10), () async {
debugPrint('_whisperTranscriptTimer triggering');
File file = await audioStorage!.createWavFile();
await transcribeAudioFile(file);
// "clear previous bytes"
// audioStorage?.clearAudioBytes();
_whisperTimer();
});
}

_initiateTimer() {
_timer?.cancel();
_timer = Timer(const Duration(seconds: 30), () async {
debugPrint('Creating memory from whispers');
String transcript = '';
if (customWebsocketTranscript.trim().isEmpty) {
transcript = customWebsocketTranscript.trim();
} else {
transcript = _buildDiarizedTranscriptMessage();
}
debugPrint('Transcript: \n$transcript');
File file = await audioStorage!.createWavFile();
String? fileName = await uploadFile(file);
processTranscriptContent(transcript, fileName);
setState(() {
whispersDiarized = [{}];
customWebsocketTranscript = '';
});
audioStorage?.clearAudioBytes();
});
}

@override
void initState() {
super.initState();
WidgetsBinding.instance.addObserver(this);
initBleConnection();
}

//
// @override
// void dispose() {
// super.dispose();
// }
@override
void didChangeAppLifecycleState(AppLifecycleState state) {
super.didChangeAppLifecycleState(state);
if (state == AppLifecycleState.paused) {
addEventToContext('App is paused');
debugPrint('App is paused');
} else if (state == AppLifecycleState.resumed) {
addEventToContext('App is resumed');
debugPrint('App is resumed');
} else if (state == AppLifecycleState.hidden) {
addEventToContext('App is hidden');
debugPrint('App is hidden');
}
}

@override
void dispose() {
WidgetsBinding.instance.removeObserver(this);
super.dispose();
}

void initBleConnection() async {
Future<void> initBleConnection() async {
SchedulerBinding.instance.addPostFrameCallback((_) async {
Tuple4<IOWebSocketChannel?, StreamSubscription?, WavBytesUtil, IOWebSocketChannel?> data = await bleReceiveWAV(
btDevice: widget.btDevice!,
Expand All @@ -133,7 +87,7 @@ class TranscriptWidgetState extends State<TranscriptWidget> with TickerProviderS
_initiateTimer();
},
interimCallback: (String transcript, Map<int, String> transcriptBySpeaker) {
_timer?.cancel();
_memoryCreationTimer?.cancel();
var copy = whispersDiarized[whispersDiarized.length - 1];
transcriptBySpeaker.forEach((speaker, transcript) {
copy[speaker] = transcript;
Expand Down Expand Up @@ -165,7 +119,11 @@ class TranscriptWidgetState extends State<TranscriptWidget> with TickerProviderS
// connection was okay, but then failed.
setState(() {
wsConnectionState = WebsocketConnectionStatus.error;
websocketReconnecting = false;
});
createNotification(
title: 'Deepgram Connection Error',
body: 'There was an error with the Deepgram connection, please reconnect.');
},
onCustomWebSocketCallback: (String transcript) async {
// debugPrint('Custom Websocket Callback: $transcript');
Expand Down Expand Up @@ -195,7 +153,7 @@ class TranscriptWidgetState extends State<TranscriptWidget> with TickerProviderS
setState(() {
whispersDiarized = [{}];
customWebsocketTranscript = '';
_timer?.cancel();
_memoryCreationTimer?.cancel();
if (resetBLEConnection) {
setState(() {
websocketReconnecting = true;
Expand All @@ -205,6 +163,97 @@ class TranscriptWidgetState extends State<TranscriptWidget> with TickerProviderS
});
}

int _reconnectionAttempts = 0;
final int _maxReconnectionAttempts = 3;

Future<void> _reconnectWebSocket() async {
if (_reconnectionAttempts >= _maxReconnectionAttempts) {
setState(() {
websocketReconnecting = false;
});
debugPrint('Max reconnection attempts reached');
return;
}

if (wsConnectionState == WebsocketConnectionStatus.notConnected ||
wsConnectionState == WebsocketConnectionStatus.closed ||
wsConnectionState == WebsocketConnectionStatus.failed ||
wsConnectionState == WebsocketConnectionStatus.error) {
setState(() {
websocketReconnecting = true;
});

_reconnectionAttempts++;
await Future.delayed(const Duration(seconds: 3)); // Reconnect delay

try {
await initBleConnection();
if (channel != null) {
channel!.sink.add('{"action":"start"}');
}
setState(() {
_reconnectionAttempts = 0; // Reset counter on successful connection
websocketReconnecting = false;
});
} catch (e) {
debugPrint('Reconnection attempt $_reconnectionAttempts failed: $e');
_reconnectWebSocket(); // Try to reconnect again
}
}
}

String _buildDiarizedTranscriptMessage() {
int totalSpeakers = whispersDiarized
.map((e) => e.keys.isEmpty ? 0 : ((e.keys).max + 1))
.reduce((value, element) => value > element ? value : element);

debugPrint('Speakers count: $totalSpeakers');

String transcript = '';
for (int partIdx = 0; partIdx < whispersDiarized.length; partIdx++) {
var part = whispersDiarized[partIdx];
if (part.isEmpty) continue;
for (int speaker = 0; speaker < totalSpeakers; speaker++) {
if (part.containsKey(speaker)) {
// This part and previous have only 1 speaker, and is the same
if (partIdx > 0 &&
whispersDiarized[partIdx - 1].containsKey(speaker) &&
whispersDiarized[partIdx - 1].length == 1 &&
part.length == 1) {
transcript += '${part[speaker]!} ';
} else {
transcript += 'Speaker $speaker: ${part[speaker]!} ';
}
}
}
transcript += '\n';
}
return transcript;
}

_initiateTimer() {
_memoryCreationTimer?.cancel();
_memoryCreationTimer = Timer(const Duration(seconds: 30), () async {
debugPrint('Creating memory from whispers');
String transcript = '';
if (customWebsocketTranscript.trim().isEmpty) {
transcript = customWebsocketTranscript.trim();
} else {
transcript = _buildDiarizedTranscriptMessage();
}
debugPrint('Transcript: \n$transcript');
File file = await audioStorage!.createWavFile();
String? fileName = await uploadFile(file);
processTranscriptContent(transcript, fileName);
addEventToContext('Memory Created');
setState(() {
whispersDiarized = [{}];
customWebsocketTranscript = '';
});
audioStorage?.clearAudioBytes();
});
}

@override
Widget build(BuildContext context) {
context.watch<FFAppState>();
Expand Down
10 changes: 5 additions & 5 deletions apps/AppWithWearable/lib/utils/sentry_log.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import 'package:sentry_flutter/sentry_flutter.dart';

addDeepgramEventContext(String event) {
addEventToContext(String event) {
// TODO: instead of doing this, include a way to add single events that can be traced and combined
// click x, bluetooth disconnected, ws disconnected, click, ws connected ... etc..
Sentry.configureScope((scope) {
var deepgramData = (scope.contexts['deepgram'] ?? {});
var currentEvents = deepgramData['events'] ?? [];
var deepgramData = (scope.contexts['events'] ?? {});
var currentEvents = deepgramData['values'] ?? [];
currentEvents.add(event);
deepgramData['events'] = currentEvents;
scope.setContexts('deepgram', deepgramData);
deepgramData['values'] = currentEvents;
scope.setContexts('events', deepgramData);
});
}
14 changes: 8 additions & 6 deletions apps/AppWithWearable/lib/utils/stt/deepgram.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,8 @@ Future<IOWebSocketChannel> _initStream(
(event) {
// debugPrint('Event from Stream: $event');
final parsedJson = jsonDecode(event);
// FIXME Receiver: null ~ Tried calling: []("alternatives")
if (parsedJson['channel'] == null || parsedJson['channel']['alternatives'] == null) return;

final data = parsedJson['channel']['alternatives'][0];
final transcript = data['transcript'];
final speechFinal = parsedJson['is_final'];
Expand All @@ -113,31 +114,32 @@ Future<IOWebSocketChannel> _initStream(
},
onError: (err) {
debugPrint('Websocket Error: $err');
addDeepgramEventContext('Websocket Error');
addEventToContext('Websocket Error');
Sentry.captureException(err, stackTrace: err.stackTrace);
onWebsocketConnectionError(err);
},
onDone: (() {
debugPrint('Websocket Closed');
addDeepgramEventContext('Websocket Closed');
addEventToContext('Websocket Closed');
Sentry.captureMessage('Websocket Closed', level: SentryLevel.warning);
onWebsocketConnectionClosed();
}),
cancelOnError: true,
);
}).onError((error, stackTrace) {
addDeepgramEventContext('Websocket Unable To Connect');
addEventToContext('Websocket Unable To Connect');
debugPrint("WebsocketChannel was unable to establish connection");
onWebsocketConnectionFailed();
});

try {
await channel.ready;
debugPrint('Websocket Opened');
addDeepgramEventContext('Websocket Opened');
addEventToContext('Websocket Opened');
onWebsocketConnectionSuccess();
} catch (err) {
debugPrint("Websocket was unable to establish connection");
addDeepgramEventContext('Websocket Unable To Connect 2');
addEventToContext('Websocket Unable To Connect 2');
onWebsocketConnectionFailed();
Sentry.captureException(err);
}
Expand Down